# -*- coding: utf-8 -*-
# @Time    : 2024/1/23 20:02
# @Author  : micah
# @File    : 9.数据去重-芒果TV数据抓取并保存MongoDB.py
# @Software: PyCharm


import redis
import pymongo
import hashlib
import requests


class MgMovieInfo:
    def __init__(self):
        self.api_url = "https://pianku.api.mgtv.com/rider/list/pcweb/v3"
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Referer': 'https://www.mgtv.com/'
        }

        self.redis_client = redis.Redis()
        self.mongo_client = pymongo.MongoClient(host='localhost', port=27017)
        self.collection = self.mongo_client['py_spider']['mg_movie_info']

    # 获取数据方法
    def get_movie_info(self, params):
        response = requests.get(self.api_url, headers=self.headers, params=params).json()
        return response['data']['hitDocs']

    # 解析数据方法
    def parse_movie_info(self, response):
        for movie in response:
            item = dict()
            item['title'] = movie['title']
            item['subtitle'] = movie['subtitle']
            item['story'] = movie['story']

            # 调用保存方法
            self.save_movie_info(item)

    # 将获取的数据打包成md5
    @staticmethod
    def get_md5(value):
        md5_hash = hashlib.md5(str(value).encode('utf-8')).hexdigest()
        return md5_hash

    # 数据保存方法
    def save_movie_info(self, item):
        md5_value = self.get_md5(item)

        # 将处理好的md5的值保存到redis中
        # 如果一个数据结构对应的key包含冒号, 则redis会创建一个类似文件夹的数据结构
        result = self.redis_client.sadd('movie:filter', md5_value)
        if result:
            self.collection.insert_one(item)
            print(f'保存成功: {item}')
        else:
            print('数据重复...')

    # 启动方法
    def main(self):
        for page in range(1, 6):
            params_dict = {
                "allowedRC": "1",
                "platform": "pcweb",
                "channelId": "2",
                "pn": str(page),
                "pc": "80",
                "hudong": "1",
                "_support": "10000000",
                "kind": "19",
                "area": "10",
                "year": "all",
                "chargeInfo": "a1",
                "sort": "c2",
                "feature": "all"
            }

            response = self.get_movie_info(params_dict)
            self.parse_movie_info(response)


if __name__ == '__main__':
    movie_info = MgMovieInfo()
    movie_info.main()
